\documentclass[fleqn]{beamer}\usepackage[]{graphicx}\usepackage[]{color}
% maxwidth is the original width if it is less than linewidth
% otherwise use linewidth (to make sure the graphics do not exceed the margin)
\makeatletter
\def\maxwidth{ %
  \ifdim\Gin@nat@width>\linewidth
    \linewidth
  \else
    \Gin@nat@width
  \fi
}
\makeatother

\definecolor{fgcolor}{rgb}{0.345, 0.345, 0.345}
\newcommand{\hlnum}[1]{\textcolor[rgb]{0.686,0.059,0.569}{#1}}%
\newcommand{\hlstr}[1]{\textcolor[rgb]{0.192,0.494,0.8}{#1}}%
\newcommand{\hlcom}[1]{\textcolor[rgb]{0.678,0.584,0.686}{\textit{#1}}}%
\newcommand{\hlopt}[1]{\textcolor[rgb]{0,0,0}{#1}}%
\newcommand{\hlstd}[1]{\textcolor[rgb]{0.345,0.345,0.345}{#1}}%
\newcommand{\hlkwa}[1]{\textcolor[rgb]{0.161,0.373,0.58}{\textbf{#1}}}%
\newcommand{\hlkwb}[1]{\textcolor[rgb]{0.69,0.353,0.396}{#1}}%
\newcommand{\hlkwc}[1]{\textcolor[rgb]{0.333,0.667,0.333}{#1}}%
\newcommand{\hlkwd}[1]{\textcolor[rgb]{0.737,0.353,0.396}{\textbf{#1}}}%
\let\hlipl\hlkwb

\usepackage{framed}
\makeatletter
\newenvironment{kframe}{%
 \def\at@end@of@kframe{}%
 \ifinner\ifhmode%
  \def\at@end@of@kframe{\end{minipage}}%
  \begin{minipage}{\columnwidth}%
 \fi\fi%
 \def\FrameCommand##1{\hskip\@totalleftmargin \hskip-\fboxsep
 \colorbox{shadecolor}{##1}\hskip-\fboxsep
     % There is no \\@totalrightmargin, so:
     \hskip-\linewidth \hskip-\@totalleftmargin \hskip\columnwidth}%
 \MakeFramed {\advance\hsize-\width
   \@totalleftmargin\z@ \linewidth\hsize
   \@setminipage}}%
 {\par\unskip\endMakeFramed%
 \at@end@of@kframe}
\makeatother

\definecolor{shadecolor}{rgb}{.97, .97, .97}
\definecolor{messagecolor}{rgb}{0, 0, 0}
\definecolor{warningcolor}{rgb}{1, 0, 1}
\definecolor{errorcolor}{rgb}{1, 0, 0}
\newenvironment{knitrout}{}{} % an empty environment to be redefined in TeX

\usepackage{alltt}

% \usefonttheme[onlylarge]{structuresmallcapsserif}
\usefonttheme[onlysmall]{structurebold}
\usepackage[normalem]{ulem} % use normalem to protect \emph
\newcommand\hl{\bgroup\markoverwith
  {\textcolor{yellow}{\rule[-.5ex]{2pt}{2.5ex}}}\ULon}
\usepackage{amsmath,amssymb}
\usepackage{booktabs}
\usepackage{graphicx}
\usepackage{longtable}
\usepackage{lscape}
\usepackage[authoryear]{natbib}
\usepackage{array}
\usepackage{listings}
\usepackage{wasysym}
\usepackage{url}
\usepackage{multirow}
\usepackage{color}
\usepackage{qtree}
\usepackage{inconsolata}
\usepackage{framed}
\definecolor{shadecolor}{rgb}{1,0.8,0.3}
\usepackage[justification=centering]{caption}
\captionsetup{compatibility=false}
\usepackage{subcaption}
\usepackage{xcolor}
\usepackage{hyperref}
\hypersetup{colorlinks=true,linkcolor=blue}
\usepackage[space]{grffile}
\usefonttheme[onlymath]{serif}
% \usepackage{tikz}
% \usetikzlibrary{calc,matrix}
\usepackage{sgame}
\usepackage{tikz}
\usetikzlibrary{trees}

\mode<presentation>
%\usetheme{Frankfurt}
\usetheme{boxes}
\usecolortheme{beaver}
%\usetheme{Singapore}
%\usetheme{Hannover}

\beamertemplatenavigationsymbolsempty
\setbeamertemplate{blocks}[rounded]
\newenvironment<>{block_code}[1]{%
  \setbeamercolor{block title}{fg=white,bg=black}%
  \begin{block}#2{#1}}{\end{block}}

%--------------------------
% Reduce the spacing between R code and output
%--------------------------
\usepackage{etoolbox}
\makeatletter
\preto{\@verbatim}{\topsep=0pt \partopsep=0pt }
\makeatother

\renewenvironment{knitrout}{\setlength{\topsep}{0mm}}{}

%===================================
% Coloring change
%===================================
\definecolor{darkred}{rgb}{0.8,0,0}
\setbeamercolor{block title}{fg=darkred,bg=structure.fg!20!bg!50!bg}
\setbeamercolor{block body}{use=block title,bg=block title.bg}

\title{Multivariate Regression}
\author{Taro Mieno}
\date{AECN 896-003: Applied Econometrics}
\everymath{\displaystyle}
\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
\begin{document}




\begin{frame}
\titlepage
\end{frame}


\begin{frame}[c,fragile]
  \frametitle{Univariate regression model}
  \begin{block}{Drawback}
    The most important assumption $E[u|x]$ is almost always violated (unless you data comes from randomized experiments)
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Multivariate regression model}
  \begin{block}{Improvement over univariate regression model}
    More independent variables mean less factors left in the error term, which makes the endogeneity problem \textcolor{blue}{less} severe
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Two independent variables}
  \begin{block}{Bi-variate vs. uni-variate}
  \vspace{-0.6cm}
    \begin{align*}
    wage = & \beta_0 + \beta_1 educ + \beta_2 exper + u_2 \\
    wage = & \beta_0 + \beta_1 educ + u_1 (=u_2+\beta_2 exper)
  \end{align*}
  \end{block}
  \begin{block}{What's different?}
    \begin{description}
      \item [bi-variate]: able to measure the effect of education on wage, \textcolor{blue}{holding experience fixed} because experience is modeled explicitly (\textcolor{red}{We say $exper$ is controlled for.})

      \item [univariate]: $\hat{\beta_1}$ is biased unless experience is uncorrelated with education because experience was in error term
    \end{description}
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Two independent variables}
  \begin{block}{Another example}
  The impact of per student spending (\textit{expend}) on standardized test score (\textit{avgscore}) at the high school level
    \begin{align*}
    avgscore= & \beta_0+\beta_1 expend + u_1 (=u_2+\beta_2 avginc) \notag \\
    avgscore= & \beta_0+\beta_1 expend +\beta_2 avginc + u_2 \notag
    \end{align*}
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Two independent variables}
  More generally,
  \begin{align}
    y=\beta_0+\beta_1 x_1 + \beta_2 x_2 + u
  \end{align}
  \begin{description}
    \item [$\beta_0$]: intercept
    \item [$\beta_1$]: measure the change in $y$ with respect to $x_1$, holding other factors fixed
    \item [$\beta_2$]: measure the change in $y$ with respect to $x_1$, holding other factors fixed
  \end{description}
\end{frame}



\begin{frame}[c]
  \frametitle{The Crucial Condition (Assumption) for Unbiasedness of OLS}
  \begin{block}{Uni-variate}
    $E[u|x]=0$
  \end{block}

  \begin{block}{Bi-variate}
  \begin{itemize}
    \item Mathematically: $E[u|x_1,x_2]=0$
    \item Verbally: for any values of $x_1$ and $x_2$, the expected value of the unobservables is zero
  \end{itemize}
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Mean independence condition}
  In the following wage model,
  \begin{align*}
    wage = & \beta_0 + \beta_1 educ + \beta_2 exper + u
  \end{align*}
  Mean independence condition is,
  \begin{align*}
   E[u|educ,exper]=0
  \end{align*}
  This condition would be satisfied if innate ability of students is on average unrelated to education level and experience.
\end{frame}

\begin{frame}[c]
  \frametitle{The model with $k$ independent variables}
  \begin{block}{Multivariate regression model (in general)}
  \vspace{-0.6cm}
     \begin{align*}
        y=\beta_0 + \beta_1 x_1 + \beta_2 x_2 + \dots + \beta_k x_k + u
     \end{align*}
   \end{block}
  \only<2->{\begin{block}{Mean independence assumption?}
  $\beta_{OLS}$ (OLS estimators of $\beta$s) is unbiased if,
    \begin{align*}
       E[u|x_1,x_2,\dots,x_k]=0
    \end{align*}
  \end{block}}
\end{frame}

\begin{frame}[c]
  \frametitle{Deriving OLS estimators}
  \begin{block}{OLS}
    Find the combination of $\beta$s that minimizes the sum of squared residuals
  \end{block}

  \begin{block}{So,}
  Denoting the collection of $\hat{\beta}$s as $\hat{\theta} (=\{\hat{\beta_0},\hat{\beta_1},\dots,\hat{\beta_k}\})$,
  \begin{align*}
     Min_{\theta} \sum_{i=1}^n \Big[ y_i-(\hat{\beta_0}+\hat{\beta_1} x_{1,i} + \hat{\beta_2} x_{2,i} + \dots + \beta_k x_{k,i}) \Big]^2
  \end{align*}
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Deriving OLS estimators}
  \only<1-1>{Find the FOCs by partially differentiating the objective function (sum of squared residuals) wrt each of $\hat{\theta} (=\{\hat{\beta_0},\hat{\beta_1},\dots,\hat{\beta_k}\})$,
  \begin{align*}
     \sum_{i=1}^n(y_i-(\hat{\beta_0}+\hat{\beta_1} x_{1,i} + \hat{\beta_2} x_{2,i} + \dots + \beta_k x_{k,i}) = & 0 \;\; (\beta_0) \\
     \sum_{i=1}^n x_{i,1}\Big[ y_i-(\hat{\beta_0}+\hat{\beta_1} x_{1,i} + \hat{\beta_2} x_{2,i} + \dots + \beta_k x_{k,i}) \Big]= & 0  \;\; (\beta_1) \\
    \sum_{i=1}^n x_{i,2}\Big[ y_i-(\hat{\beta_0}+\hat{\beta_1} x_{1,i} + \hat{\beta_2} x_{2,i} + \dots + \beta_k x_{k,i}) \Big]= & 0  \;\; (\beta_2) \\
    \vdots \\
    \sum_{i=1}^n x_{i,k}\Big[ y_i-(\hat{\beta_0}+\hat{\beta_1} x_{1,i} + \hat{\beta_2} x_{2,i} + \dots + \beta_k x_{k,i}) \Big]= & 0  \;\; (\beta_k) \\
  \end{align*}}
  \only<2->{
  Or more succinctly,
  \begin{align*}
    \sum_{i=1}^n \hat{u}_i = & 0 \;\; (\beta_0) \\
    \sum_{i=1}^n x_{i,1}\hat{u}_i = & 0  \;\; (\beta_1) \\
    \sum_{i=1}^n x_{i,2}\hat{u}_i = & 0  \;\; (\beta_2) \\
    \vdots \\
    \sum_{i=1}^n x_{i,k}\hat{u}_i = & 0  \;\; (\beta_k) \\
  \end{align*}
  }
\end{frame}

\begin{frame}[c,fragile]
  \frametitle{Implementation of multivariate OLS}
  \begin{block_code}{R code: Implementation in R}
  \footnotesize{
\begin{knitrout}
\definecolor{shadecolor}{rgb}{0.969, 0.969, 0.969}\color{fgcolor}\begin{kframe}
\begin{alltt}
  \hlcom{#--- generate data ---#}
  \hlstd{N} \hlkwb{<-} \hlnum{100} \hlcom{# sample size}
  \hlstd{x1} \hlkwb{<-} \hlkwd{rnorm}\hlstd{(N)} \hlcom{# independent variable}
  \hlstd{x2} \hlkwb{<-} \hlkwd{rnorm}\hlstd{(N)} \hlcom{# independent variable}
  \hlstd{u} \hlkwb{<-} \hlkwd{rnorm}\hlstd{(N)} \hlcom{# error}
  \hlstd{y} \hlkwb{<-} \hlnum{1} \hlopt{+} \hlstd{x1} \hlopt{+} \hlstd{x2} \hlopt{+} \hlstd{u} \hlcom{# dependent variable}
  \hlstd{data} \hlkwb{<-} \hlkwd{data.frame}\hlstd{(}\hlkwc{y}\hlstd{=y,}\hlkwc{x1}\hlstd{=x1,}\hlkwc{x2}\hlstd{=x2)}

  \hlcom{#--- OLS ---#}
  \hlstd{reg} \hlkwb{<-} \hlkwd{lm}\hlstd{(y}\hlopt{~}\hlstd{x1}\hlopt{+}\hlstd{x2,}\hlkwc{data}\hlstd{=data)}
  \hlstd{reg_sum} \hlkwb{<-} \hlkwd{summary}\hlstd{(reg)} \hlcom{# get summary}
  \hlstd{reg_sum}\hlopt{$}\hlstd{coef} \hlcom{# print out coef estimates}
\end{alltt}
\begin{verbatim}
            Estimate Std. Error  t value     Pr(>|t|)
(Intercept) 1.010631 0.09123388 11.07736 6.448002e-19
x1          1.085433 0.07576744 14.32586 1.125846e-25
x2          1.101331 0.09299542 11.84285 1.510965e-20
\end{verbatim}
\end{kframe}
\end{knitrout}
  }
  \end{block_code}
\end{frame}


\begin{frame}[c]
  \frametitle{A partialling out interpretation}
  Consider the following simple model,
  \begin{align*}
    y_i = \beta_0 + \beta_1 x_{1,i} + \beta_2 x_{2,i} + \beta_3 x_{3,i} + u_i
  \end{align*}
  Suppose you are interested in estimating $\beta_1$.

\end{frame}

\begin{frame}[c]
  \frametitle{A partialling out interpretation}
Let's consider the following two methods,
  \begin{block}{Method 1: Regular OLS}
    regress $y$ on $x_1$, $x_2$, and $x_3$ with an intercept to estimate $\beta_0$, $\beta_1$, $\beta_2$, $\beta_3$ at the same time (just like you normally do)
  \end{block}
  \begin{block}{Method 2: 3-step}
    \begin{enumerate}
      \item regress $y$ on $x_2$ and $x_3$ with an intercept and get residuals, which we call $\hat{u}_y$
      \item regress $x_1$ on $x_2$ and $x_3$ with an intercept and get residuals, which we call $\hat{u}_{x_1}$
      \item regress $\hat{u}_y$ on $\hat{u}_{x_1}$ ($\hat{u}_y=\alpha_1 \hat{u}_{x_1}+v_3$)
    \end{enumerate}
  \end{block}

\end{frame}



\begin{frame}[c]
  \frametitle{A partialling out interpretation}
 \begin{block}{Frisch-–Waugh-–Lovell theorem}
 Methods 1 and 2 produces the same coefficient estimate on $x_1$
  \begin{align*}
     \hat{\beta_1} = \hat{\alpha_1}
  \end{align*}
  \end{block}
\end{frame}

\begin{frame}[c,fragile]
  \frametitle{A partialling out interpretation: Demonstration using R}
  \begin{block_code}{R code: FWL theorem}
  \footnotesize{
\begin{knitrout}
\definecolor{shadecolor}{rgb}{0.969, 0.969, 0.969}\color{fgcolor}\begin{kframe}
\begin{alltt}
  \hlcom{#--- data generation  ---#}
  \hlstd{N} \hlkwb{<-} \hlnum{100} \hlcom{# sample size}
  \hlstd{x1} \hlkwb{<-} \hlkwd{rnorm}\hlstd{(N)} \hlcom{# independent variable}
  \hlstd{x2} \hlkwb{<-} \hlkwd{rnorm}\hlstd{(N)} \hlcom{# independent variable}
  \hlstd{u} \hlkwb{<-} \hlkwd{rnorm}\hlstd{(N)} \hlcom{# error}
  \hlstd{y} \hlkwb{<-} \hlnum{1} \hlopt{+} \hlstd{x1} \hlopt{+} \hlstd{x2} \hlopt{+} \hlstd{u} \hlcom{# dependent variable}
  \hlstd{data} \hlkwb{<-} \hlkwd{data.frame}\hlstd{(}\hlkwc{y}\hlstd{=y,}\hlkwc{x1}\hlstd{=x1,}\hlkwc{x2}\hlstd{=x2)}

  \hlcom{#--- method 1 (one-step) ---#}
  \hlstd{beta_m1} \hlkwb{<-} \hlkwd{lm}\hlstd{(y}\hlopt{~}\hlstd{x1}\hlopt{+}\hlstd{x2,}\hlkwc{data}\hlstd{=data)}\hlopt{$}\hlstd{coef[}\hlstr{'x1'}\hlstd{]} \hlcom{# OLS}

  \hlcom{#--- method 3 (three-step) ---#}
  \hlstd{y_res} \hlkwb{<-} \hlkwd{lm}\hlstd{(y}\hlopt{~}\hlstd{x2,}\hlkwc{data}\hlstd{=data)}\hlopt{$}\hlstd{residuals} \hlcom{# OLS}
  \hlstd{x1_res} \hlkwb{<-} \hlkwd{lm}\hlstd{(x1}\hlopt{~}\hlstd{x2,}\hlkwc{data}\hlstd{=data)}\hlopt{$}\hlstd{residuals} \hlcom{# OLS}
  \hlstd{beta_m2} \hlkwb{<-} \hlkwd{lm}\hlstd{(y}\hlopt{~}\hlstd{x,}\hlkwc{data}\hlstd{=}\hlkwd{data.frame}\hlstd{(}\hlkwc{y}\hlstd{=y_res,}\hlkwc{x}\hlstd{=x1_res))}\hlopt{$}\hlstd{coef[}\hlstr{'x'}\hlstd{]}
\end{alltt}
\end{kframe}
\end{knitrout}
  }
  \end{block_code}
\end{frame}

\begin{frame}[c]
  \frametitle{What does this mean?}
  \begin{block}{Method 2: 3-step}
    \begin{enumerate}
      \item $y_i=\gamma_0+\gamma_2 x_2 + \gamma_3 x_3 + v_1$
      \item $x_i=\delta_0+\delta_2 x_2 + \delta_3 x_3 + v_2$
      \item $\hat{v}_1=\alpha_1 \hat{v}_2+v_3$
    \end{enumerate}
  \end{block}
  \begin{block}{Partialling out}
  \begin{itemize}
    \item $\hat{\beta_1}$ is the impact of $x_1$ with the impacts of the other variables \textcolor{blue}{netted out}!!
    \item including other variables (take them out from the error term)
  \end{itemize}
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Unbiasedness of OLS Estimators}
  \begin{block}{Unbiasedness}
    OLS estimators of multivariate models are unbiased under \textcolor{blue}{certain} conditions
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Unbiasedness of OLS Estimators}
  \begin{block}{Condition 1}
    Your model is correct (Assumption $MLR.1$)
  \end{block}
  \begin{block}{Condition 2}
    Random sampling (Assumption $MLR.2$)
  \end{block}
  \begin{block}{Conditions 3}
    No perfect collinearity (Assumption $MLR.3$)
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Perfect Collinearity}
  \begin{block}{No Perfect Collinearity}
    Any variable cannot be a linear function of the other variables
  \end{block}
  \begin{block}{Example (silly)}
  \vspace{-0.6cm}
  \begin{align*}
     wage = \beta_0 + \beta_1 educ + \beta_2 (3\times educ) + u
  \end{align*}
  (\textcolor{blue}{More on this later when we talk about dummy variables})
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Unbiasedness of OLS Estimators}
  \begin{block}{Zero Conditional Mean}
  \vspace{-0.6cm}
     \begin{align*}
         E[u|x_1,x_2,\dots,x_k]=0 \;\;\mbox{(Assumption $MLR.4$)}
     \end{align*}
  \end{block}
  \begin{block}{Unbiasedness of OLS estimators}
    If all the conditions $MLR.1\sim MLR.4$ are satisfied, OLS estimators are unbiased.
    \begin{align*}
      E[\hat{\beta}_j]=\beta_j \;\; ^\forall j=0,1,\dots,k
    \end{align*}
  \end{block}
\end{frame}


\begin{frame}[c]
  \frametitle{Endogeneity ($E[u|x_1,x_2,\dots,x_k]\ne 0$)}
  \begin{block}{What could cause endogeneity problem?}
    \begin{itemize}
      \item functional form misspecification
      \begin{align*}
        wage = & \beta_0 + \beta_1 log(x_1) + \beta_2 x_2 + u_1 \;\;\mbox{(true)}\\
        wage = & \beta_0 + \beta_1 x_1 + \beta_2 x_2 + u_2 (=log(x_1)-x_1) \;\; \mbox{(yours)}
      \end{align*}
      \item omission of variables that are correlated with any of $x_1,x_2,\dots,x_k$ (\textcolor{blue}{more on this soon})
      \item \textcolor{blue}{other sources of enfogeneity later}
    \end{itemize}
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Variance of the OLS estimators}
  \begin{block}{Homoeskedasticity}
  \vspace{-0.6cm}
    \begin{align*}
      Var(u|x_1,\dots,x_k)=\sigma^2 \;\;\mbox{(Assumption $MLR.5$)}
    \end{align*}
  \end{block}

  \begin{block}{Variance}
    Under conditions $MLR.1$ through $MLR.5$, conditional on the sample values of the independent variables,
    \begin{align*}
       Var(\hat{\beta}_j)= \frac{\sigma^2}{SST_j(1-R^2_j)},
    \end{align*}
    where $SST_j= \sum_{i=1}^n (x_{ji}-\bar{x_j})^2$ and $R_j^2$ is the R-squared from regressing $x_j$ on all other independent variables including an intercept. (\textcolor{blue}{We will revisit this equation})
  \end{block}
\end{frame}

\begin{frame}[c]
  \frametitle{Estimating $\sigma^2$}
  Just like uni-variate regression, you need to estimate $\sigma^2$ if you want to estimate the variance (and standard deviation) of the OLS estimators.

  \begin{block}{uni-variate regression}
  \vspace{-0.6cm}
    \begin{align*}
      \hat{\sigma}^2=\sum_{i=1}^b \frac{\hat{u}_i^2}{n-\textcolor{blue}{2}}
    \end{align*}
  \end{block}

  \begin{block}{multi-variate regression (\textcolor{blue}{k} independent variables with intercept)}
  \vspace{-0.6cm}
    \begin{align*}
      \hat{\sigma}^2=\sum_{i=1}^b \frac{\hat{u}_i^2}{n-\textcolor{blue}{(k+1)}}
    \end{align*}
    You solved $k+1$ simultaneous equations to get $\hat{\beta}_j$ ($j=0,\dots,k$). So, once you know the value of $n-k-1$ of the residuals, you know the rest.
  \end{block}
\end{frame}

% \begin{frame}[c]
%   \frametitle{Gauss-Markov Theorem}
%   \begin{block}{Gauss-Markov Theorem}
%     Under conditions $MLR.1$ through $MLR.5$, OLS estimators are the best linear unbiased estimators (BLUEs)
%   \end{block}
%   \begin{block}{Meaning,}
%     No other \textcolor{blue}{unbiased linear}  estimators have smaller variance than the OLS estimators
%   \end{block}
% \end{frame}



\end{document}

